docker rm -f vllm-qwen3-1  && docker run -it --rm  --gpus all \
--privileged \
--network host \
--ipc=host \
--name vllm-qwen3-1 \
-v /data2:/data2 \
--ulimit memlock=-1 \
--entrypoint bash \
lmcache/vllm-openai:v0.3.6
m.daocloud.io/docker.io/lmsysorg/sglang:v0.4.9.post2-cu126


docker run -itd --rm --gpus all \
  --privileged \
  --network host \
  --ipc=host \
  --name sglang-bench-test \
  -v /data2:/data2 \
  --ulimit memlock=-1 \
  --entrypoint bash \
  m.daocloud.io/docker.io/lmsysorg/sglang:v0.4.9.post2-cu126

docker run -itd --rm --gpus all \
  --privileged \
  --network host \
  --ipc=host \
  --name  aiconfigrator-jiaozh3 \
  -v /data2:/data2 \
  -v /data1:/data1 \
  --ulimit memlock=-1 \
  --entrypoint bash \
  artifacts.lenovo.com/cybertron-dev/tensorrtllm-runtime:0.5.1




docker run -itd --rm --gpus all     --privileged     --network host     --ipc=host     --name vllm-qwen3-jzh-1     -v /data2:/data2     --ulimit memlock=-1     --entrypoint bash     m.daocloud.io/docker.io/vllm/vllm-openai:v0.10.0     lmcache/vllm-openai:v0.3.6

docker run -itd --rm --gpus all     --privileged     --network host     --ipc=host     --name sglang-qwen3-jzh-1     -v /data2:/data2     --ulimit memlock=-1     --entrypoint bash m.daocloud.io/docker.io/lmsysorg/sglang:v0.4.9.post2-cu126



python3 -m sglang.bench_serving --base-url http://10.121.129.6:9000  --backend vllm --dataset-name random --max-concurrency 10 --num-prompts 50 --random-input-len 200 --random-output-len 200 --model /data2/DeepSeek-R1-Distill-Qwen-32B --dataset-path /data2/datasets/ShareGPT_V3_unfiltered_cleaned_split.json;

LMCACHE_CONFIG_FILE=/data2/jzh/p2p/lm1.yaml \
LMCACHE_USE_EXPERIMENTAL=True \
CUDA_VISIBLE_DEVICES=0,1 \
NCCL_DEBUG=INFO \
/opt/venv/bin/vllm serve /data2/DeepSeek-R1-Distill-Qwen-32B \
    --trust-remote-code \
    --tensor-parallel-size 2 \
    --host 0.0.0.0 \
    --gpu-memory-utilization 0.78 \
    --port 9000 \
    --kv-transfer-config '{"kv_connector":"LMCacheConnectorV1", "kv_role":"kv_both"}' \
    --max-model-len 13000 \
    --enforce-eager \
    --no-enable-prefix-caching

python3 -m sglang.bench_serving \
    --base-url http://10.121.129.13:9000 \
    --backend vllm \
    --dataset-name random \
    --max-concurrency 10 \
    --num-prompts 50 \
    --random-input-len 200 \
    --random-output-len 200 \
    --model /data1/DeepSeek-R1-Distill-Qwen-32B \
    --dataset-path /data1/datasets/ShareGPT_V3_unfiltered_cleaned_split.json;

docker run -itd --rm --gpus all \
  --privileged \
  --network host \
  --ipc=host \
  --name lmcache-jzh \
  -v /data2:/data2 \
  -v /data1:/data1 \
  --ulimit memlock=-1 \
  --entrypoint bash \
  lmcache/vllm-openai:v0.3.7

PYTHONHASHSEED=0 LMCACHE_CONFIG_FILE=/data1/jzh/p2p/lm1.yaml LMCACHE_USE_EXPERIMENTAL=True CUDA_VISIBLE_DEVICES=0,1 NCCL_DEBUG=INFO /opt/venv/bin/vllm serve /data1/DeepSeek-R1-Distill-Qwen-32B     --trust-remote-code     --tensor-parallel-size 2     --host 0.0.0.0     --gpu-memory-utilization 0.78     --port 9001     --kv-transfer-config '{"kv_connector":"LMCacheConnectorV1", "kv_role":"kv_both"}'     --max-model-len 13000     --enforce-eager     --no-enable-prefix-caching



docker run -itd --rm --gpus all \
  --privileged \
  --network host \
  --ipc=host \
  --name dynamo-trtllm-jzh \
  -v /data2:/data2 \
  -v /data1:/data1 \
  --ulimit memlock=-1 \
  --entrypoint bash \
  nvcr.io/nvidia/ai-dynamo/trtllm-runtime:0.5.1
  m.daocloud.io/docker.io/lmsysorg/sglang:v0.4.10.post2-cu126


curl -X POST http://localhost:9001/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "DeepSeek-R1-Distill-Qwen-32B",
    "messages": [
      {"role": "user", "content": "请简单介绍一下你自己"}
    ],
    "temperature": 0.7,
    "max_tokens": 200
  }'


curl http://localhost:9001/generate \
    -H "Content-Type: application/json" \
    -d '{
        "model": "data1/DeepSeek-R1-Distill-Qwen-32B",
        "prompt": "who are you?",
        "max_tokens": 7,
        "temperature": 0
    }'

python3 -m sglang.launch_server --model-path /models/public-models/Qwen3-235B-A22B \
--tensor-parallel-size 8  --host 0.0.0.0 --port 9000 \
--trust-remote-code --mem-fraction-static 0.9
                --enable-hierarchical-cache \
                --hicache-size 50 



NCCL_DEBUG=TRACE python3 -m sglang.launch_server \
  --model-path /data2/0wen3-235B-A22B-FP8-Block32 \
  --tensor-parallel-size 16 \
  --host 0.0.0.0 \
  --port 8080 \
  --trust-remote-code \
  --mem-fraction-static 0.85 \
  --dist-init-addr 10.121.129.6:5000 \
  --nnodes 2 \
  --node-rank 0


docker run \
  -itd \
  --rm \
  --gpus all \
  --privileged \
  --network host \
  --ipc=host \
  --name aiforce-jiaozh3 \
  -v /data1:/data1 \
  -v /data2:/data2 \
  --ulimit memlock=-1 \
  --entrypoint bash \
  artifacts.lenovo.com/cybertron-dev/sglang/sglang:v0.5.5.post2




server_args=ServerArgs(model_path='/data2/Qwen3-32B-FP8', tokenizer_path='/data2/Qwen3-32B-FP8', tokenizer_mode='auto', tokenizer_worker_num=1, skip_tokenizer_init=False, load_format='auto', model_loader_extra_config='{}', trust_remote_code=False, context_length=65536, is_embedding=False, enable_multimodal=None, revision=None, model_impl='auto', host='0.0.0.0', port=8888, grpc_mode=False, skip_server_warmup=False, warmups=None, nccl_port=None, checkpoint_engine_wait_weights_before_ready=False, dtype='auto', quantization=None, quantization_param_path=None, kv_cache_dtype='auto', enable_fp32_lm_head=False, modelopt_quant=None, modelopt_checkpoint_restore_path=None, modelopt_checkpoint_save_path=None, modelopt_export_path=None, quantize_and_serve=False, mem_fraction_static=0.9, max_running_requests=256, max_queued_requests=None, max_total_tokens=None, chunked_prefill_size=4096, max_prefill_tokens=16384, schedule_policy='fcfs', enable_priority_scheduling=False, abort_on_priority_when_disabled=False, schedule_low_priority_values_first=False, priority_scheduling_preemption_threshold=10, schedule_conservativeness=1.0, page_size=1, hybrid_kvcache_ratio=None, swa_full_tokens_ratio=0.8, disable_hybrid_swa_memory=False, radix_eviction_policy='lru', device='cuda', tp_size=2, pp_size=1, pp_max_micro_batch_size=None, stream_interval=1, stream_output=False, random_seed=89550901, constrained_json_whitespace_pattern=None, constrained_json_disable_any_whitespace=False, watchdog_timeout=300, dist_timeout=None, download_dir=None, base_gpu_id=0, gpu_id_step=1, sleep_on_idle=False, log_level='info', log_level_http=None, log_requests=False, log_requests_level=2, crash_dump_folder=None, show_time_cost=False, enable_metrics=False, enable_metrics_for_all_schedulers=False, tokenizer_metrics_custom_labels_header='x-custom-labels', tokenizer_metrics_allowed_custom_labels=None, bucket_time_to_first_token=None, bucket_inter_token_latency=None, bucket_e2e_request_latency=None, collect_tokens_histogram=False, prompt_tokens_buckets=None, generation_tokens_buckets=None, gc_warning_threshold_secs=0.0, decode_log_interval=40, enable_request_time_stats_logging=False, kv_events_config=None, enable_trace=False, otlp_traces_endpoint='localhost:4317', api_key=None, served_model_name='/data2/Qwen3-32B-FP8', weight_version='default', chat_template=None, completion_template=None, file_storage_path='sglang_storage', enable_cache_report=False, reasoning_parser='qwen3', tool_call_parser=None, tool_server=None, sampling_defaults='model', dp_size=1, load_balance_method='round_robin', load_watch_interval=0.1, prefill_round_robin_balance=False, dist_init_addr=None, nnodes=1, node_rank=0, json_model_override_args='{}', preferred_sampling_params=None, enable_lora=None, max_lora_rank=None, lora_target_modules=None, lora_paths=None, max_loaded_loras=None, max_loras_per_batch=8, lora_eviction_policy='lru', lora_backend='csgmv', max_lora_chunk_size=16, attention_backend='flashinfer', decode_attention_backend=None, prefill_attention_backend=None, sampling_backend='flashinfer', grammar_backend='xgrammar', mm_attention_backend=None, nsa_prefill_backend='flashmla_sparse', nsa_decode_backend='fa3', speculative_algorithm=None, speculative_draft_model_path=None, speculative_draft_model_revision=None, speculative_draft_load_format=None, speculative_num_steps=None, speculative_eagle_topk=None, speculative_num_draft_tokens=None, speculative_accept_threshold_single=1.0, speculative_accept_threshold_acc=1.0, speculative_token_map=None, speculative_attention_mode='prefill', speculative_moe_runner_backend=None, speculative_ngram_min_match_window_size=1, speculative_ngram_max_match_window_size=12, speculative_ngram_min_bfs_breadth=1, speculative_ngram_max_bfs_breadth=10, speculative_ngram_match_type='BFS', speculative_ngram_branch_length=18, speculative_ngram_capacity=10000000, ep_size=1, moe_a2a_backend='none', moe_runner_backend='auto', flashinfer_mxfp4_moe_precision='default', enable_flashinfer_allreduce_fusion=False, deepep_mode='auto', ep_num_redundant_experts=0, ep_dispatch_algorithm='static', init_expert_location='trivial', enable_eplb=False, eplb_algorithm='auto', eplb_rebalance_num_iterations=1000, eplb_rebalance_layers_per_chunk=None, eplb_min_rebalancing_utilization_threshold=1.0, expert_distribution_recorder_mode=None, expert_distribution_recorder_buffer_size=1000, enable_expert_distribution_metrics=False, deepep_config=None, moe_dense_tp_size=None, elastic_ep_backend=None, mooncake_ib_device=None, max_mamba_cache_size=None, mamba_ssm_dtype='float32', mamba_full_memory_ratio=0.9, enable_hierarchical_cache=False, hicache_ratio=2.0, hicache_size=0, hicache_write_policy='write_through', hicache_io_backend='kernel', hicache_mem_layout='layer_first', hicache_storage_backend=None, hicache_storage_prefetch_policy='best_effort', hicache_storage_backend_extra_config=None, enable_lmcache=False, kt_weight_path=None, kt_method='AMXINT4', kt_cpuinfer=None, kt_threadpool_count=2, kt_num_gpu_experts=None, kt_max_deferred_experts_per_token=None, enable_double_sparsity=False, ds_channel_config_path=None, ds_heavy_channel_num=32, ds_heavy_token_num=256, ds_heavy_channel_type='qk', ds_sparse_decode_threshold=4096, cpu_offload_gb=0, offload_group_size=-1, offload_num_in_group=1, offload_prefetch_step=1, offload_mode='cpu', multi_item_scoring_delimiter=None, disable_radix_cache=False, cuda_graph_max_bs=32, cuda_graph_bs=[1, 2, 4, 8, 12, 16, 24, 32], disable_cuda_graph=False, disable_cuda_graph_padding=False, enable_profile_cuda_graph=False, enable_cudagraph_gc=False, enable_nccl_nvls=False, enable_symm_mem=False, disable_flashinfer_cutlass_moe_fp4_allgather=False, enable_tokenizer_batch_encode=False, disable_tokenizer_batch_decode=False, disable_outlines_disk_cache=False, disable_custom_all_reduce=False, enable_mscclpp=False, enable_torch_symm_mem=False, disable_overlap_schedule=False, enable_mixed_chunk=False, enable_dp_attention=False, enable_dp_lm_head=False, enable_two_batch_overlap=False, enable_single_batch_overlap=False, tbo_token_distribution_threshold=0.48, enable_torch_compile=False, enable_piecewise_cuda_graph=False, torch_compile_max_bs=32, piecewise_cuda_graph_max_tokens=4096, piecewise_cuda_graph_tokens=[4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 640, 768, 896, 1024, 1152, 1280, 1408, 1536, 1664, 1792, 1920, 2048, 2176, 2304, 2432, 2560, 2688, 2816, 2944, 3072, 3200, 3328, 3456, 3584, 3712, 3840, 3968, 4096], piecewise_cuda_graph_compiler='eager', torchao_config='', enable_nan_detection=False, enable_p2p_check=False, triton_attention_reduce_in_fp32=False, triton_attention_num_kv_splits=8, triton_attention_split_tile_size=None, num_continuous_decode_steps=1, delete_ckpt_after_loading=False, enable_memory_saver=False, enable_weights_cpu_backup=False, allow_auto_truncate=False, enable_custom_logit_processor=False, flashinfer_mla_disable_ragged=False, disable_shared_experts_fusion=False, disable_chunked_prefix_cache=False, disable_fast_image_processor=False, keep_mm_feature_on_device=False, enable_return_hidden_states=False, scheduler_recv_interval=1, numa_node=None, enable_deterministic_inference=False, rl_on_policy_target=None, enable_dynamic_batch_tokenizer=False, dynamic_batch_tokenizer_batch_size=32, dynamic_batch_tokenizer_batch_timeout=0.002, debug_tensor_dump_output_folder=None, debug_tensor_dump_layers=None, debug_tensor_dump_input_file=None, debug_tensor_dump_inject=False, disaggregation_mode='null', disaggregation_transfer_backend='mooncake', disaggregation_bootstrap_port=8998, disaggregation_decode_tp=None, disaggregation_decode_dp=None, disaggregation_prefill_pp=1, disaggregation_ib_device=None, disaggregation_decode_enable_offload_kvcache=False, num_reserved_decode_tokens=512, disaggregation_decode_polling_interval=1, custom_weight_loader=[], weight_loader_disable_mmap=False, remote_instance_weight_loader_seed_instance_ip=None, remote_instance_weight_loader_seed_instance_service_port=None, remote_instance_weight_loader_send_weights_group_ports=None, enable_pdmux=False, pdmux_config_path=None, sm_group_num=8, mm_max_concurrent_calls=32, mm_per_request_timeout=10.0, decrypted_config_file=None, decrypted_draft_config_file=None)
[2025-11-17 06:32:22] Warning: User-specified context_length (65536) is greater than the derived context_length (40960). This may lead to incorrect model outputs or CUDA errors. Note that the derived context_length may differ from max_position_embeddings in the model's config.